# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
from .utils.itemtools import *
from itemloaders import ItemLoader
from itemloaders.processors import Compose, MapCompose, Join, Identity


class TlnewsspiderItem(scrapy.Item):
    # define the fields for your item here like:
    uuid = scrapy.Field()
    title = scrapy.Field()
    subtitle = scrapy.Field()
    summary = scrapy.Field()
    author = scrapy.Field()
    site_name = scrapy.Field()
    site_url = scrapy.Field()
    source_url = scrapy.Field()
    article_source = scrapy.Field()
    company_code = scrapy.Field()
    company_name = scrapy.Field()
    publish_date = scrapy.Field()
    spider_time = scrapy.Field()
    created_time = scrapy.Field()
    classification = scrapy.Field()
    data_from = scrapy.Field()
    content_text = scrapy.Field()
    other = scrapy.Field()


class TlnewsItemLoader(ItemLoader):
    default_output_processor = Compose(Strip(), TakeFirst())
    content_text_out = Compose(Strip(), Join(), ReplaceWhiteSpaceCharacter())
    publish_date_out = Compose(Strip(), TakeFirst())
    author_out = AuthorOrJournalFilter()
    title_out = Compose(TakeFirst(), ReplaceWhiteSpaceCharacter())


# --------不同类型网站数据存储定义------------------

# 帖子
class PostNewsItem(scrapy.Item):
    _id = scrapy.Field() # 爬虫数据库唯一ID
    id = scrapy.Field() # 爬虫数据库唯一ID
    uuid = scrapy.Field() #内容唯一键
    content	= scrapy.Field() #帖子正文
    originalContent	= scrapy.Field() #帖子原文
    source = scrapy.Field() #资讯源
    subSource = scrapy.Field() #二级资讯源（可以为空）
    originalUrl	= scrapy.Field() #原文url
    images = scrapy.Field() #正文里的图片URL列表
    spider_time = scrapy.Field()

# 资讯
class InfoNewsItem(scrapy.Item):
    _id = scrapy.Field() #爬虫数据库唯一ID
    id = scrapy.Field() # 爬虫数据库唯一ID
    uuid = scrapy.Field() #内容唯一键
    title = scrapy.Field() #标题
    originalTitle = scrapy.Field() #原文标题
    content = scrapy.Field() #正文
    images = scrapy.Field(default=[]) #正文里的图片URL列表
    source = scrapy.Field() #资讯源
    originalUrl = scrapy.Field() #原文URL
    spider_time = scrapy.Field()


# 快讯
class QuickNewsItem(scrapy.Item):
    _id = scrapy.Field() #爬虫数据库唯一ID
    id = scrapy.Field() # 爬虫数据库唯一ID
    uuid = scrapy.Field() #内容唯一键
    title = scrapy.Field() #标题
    content = scrapy.Field() #内容
    originalContent = scrapy.Field() #原文内容
    source = scrapy.Field() #资讯源
    originalUrl = scrapy.Field() #原文URL
    spider_time = scrapy.Field()
    images = scrapy.Field(default=[]) #正文里的图片URL列表



# -----------Item 处理器----------------
class newsItemLoader(ItemLoader):
    default_output_processor = Compose(Strip(), TakeFirst())
    content_out = Compose(Strip(), Join())
    images_out = Compose(Strip())

